#! /usr/bin/perl/ -w
use strict;
use warnings;
use LWP::Simple;
use DBI;

#given an url, return page content
sub getPageContent{
	my $url = shift;
	my $page=get($url);
	return $page;
}


# given a page content, return all links in <a> tag without nofollow
sub getAllListLinks{
	my $content = shift;
	my @allLinks = $content =~ /<h2>.*?<a.*?href\s*=\s*"(http[^"\s\>]+)/gis;
	my @result=();
	my %existedHash=();
	$existedHash{$_}++;
	for(my $i=0; $i<scalar(@allLinks); $i++){
		my $stringIndex = index($allLinks[$i], "searchenginewatch");
		if($stringIndex<=0){
			next;
		}
		#print $pageUrls[0]."\n";
		my $pageUrl = $allLinks[$i];
		if(exists($existedHash{$pageUrl})){
                        next;
                }
		#print $pageUrl."\n";
		$existedHash{$pageUrl}++;
		my $pageContent = getPageContent($pageUrl);
		print $pageUrl."\t".extractTitle($pageContent)."\n";
		print $pageUrl."\t".extractContent($pageContent)."\n";
		sleep(2);
		push(@result, $pageUrl);
	}
	my $resultStr = join("\n", @result);
	return $resultStr;
}
sub extractTitle{
	my $content = shift;
	my @matchTitle = $content =~ /<h1 itemprop="name">(.*?)<\/h1>/gis;
	return $matchTitle[0];
}
sub extractContent{
	my $content = shift;
	#begin to extract content
	my @matchContent = $content =~ /<div class="ukn-article-content" itemprop="articleBody">(.*?)<hr style="size:1px;color:#ddd">/gis;
	my $extractContent = $matchContent[0];
	$extractContent =~ s/<.*?>//g;
	$extractContent =~ s/&nbsp;//g;
	#$extractContent =~ s/\s+/\s/g;
	$extractContent =~ s/\n+\s+/\n/g;
	$extractContent =~ s/\s+\n+/\n/g;
	$extractContent =~ s/\n+/\n/g;
	return $extractContent;
}
#test code bellow
my $pageContent = getPageContent("http://searchenginewatch.com");
print "test".$pageContent."\n";
print getAllListLinks($pageContent)."\n";

1;
